{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network Example\n",
    "\n",
    "Build a 2-hidden layers fully connected neural network (a.k.a multilayer perceptron) with TensorFlow v2.\n",
    "\n",
    "This example is using a low-level approach to better understand all mechanics behind building neural networks and the training process.\n",
    "\n",
    "- Author: Aymeric Damien\n",
    "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network Overview\n",
    "\n",
    "<img src=\"http://cs231n.github.io/assets/nn1/neural_net2.jpeg\" alt=\"nn\" style=\"width: 400px;\"/>\n",
    "\n",
    "## MNIST Dataset Overview\n",
    "\n",
    "This example is using MNIST handwritten digits. The dataset contains 60,000 examples for training and 10,000 examples for testing. The digits have been size-normalized and centered in a fixed-size image (28x28 pixels) with values from 0 to 255. \n",
    "\n",
    "In this example, each image will be converted to float32, normalized to [0, 1] and flattened to a 1-D array of 784 features (28*28).\n",
    "\n",
    "![MNIST Dataset](http://neuralnetworksanddeeplearning.com/images/mnist_100_digits.png)\n",
    "\n",
    "More info: http://yann.lecun.com/exdb/mnist/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MNIST dataset parameters.\n",
    "num_classes = 10 # total classes (0-9 digits).\n",
    "num_features = 784 # data features (img shape: 28*28).\n",
    "\n",
    "# Training parameters.\n",
    "learning_rate = 0.001\n",
    "training_steps = 3000\n",
    "batch_size = 256\n",
    "display_step = 100\n",
    "\n",
    "# Network parameters.\n",
    "n_hidden_1 = 128 # 1st layer number of neurons.\n",
    "n_hidden_2 = 256 # 2nd layer number of neurons."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare MNIST data.\n",
    "from tensorflow.keras.datasets import mnist\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "# Convert to float32.\n",
    "x_train, x_test = np.array(x_train, np.float32), np.array(x_test, np.float32)\n",
    "# Flatten images to 1-D vector of 784 features (28*28).\n",
    "x_train, x_test = x_train.reshape([-1, num_features]), x_test.reshape([-1, num_features])\n",
    "# Normalize images value from [0, 255] to [0, 1].\n",
    "x_train, x_test = x_train / 255., x_test / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use tf.data API to shuffle and batch data.\n",
    "train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
    "train_data = train_data.repeat().shuffle(5000).batch(batch_size).prefetch(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "\n",
    "# A random value generator to initialize weights.\n",
    "random_normal = tf.initializers.RandomNormal()\n",
    "\n",
    "weights = {\n",
    "    'h1': tf.Variable(random_normal([num_features, n_hidden_1])),\n",
    "    'h2': tf.Variable(random_normal([n_hidden_1, n_hidden_2])),\n",
    "    'out': tf.Variable(random_normal([n_hidden_2, num_classes]))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.zeros([n_hidden_1])),\n",
    "    'b2': tf.Variable(tf.zeros([n_hidden_2])),\n",
    "    'out': tf.Variable(tf.zeros([num_classes]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create model.\n",
    "def neural_net(x):\n",
    "    # Hidden fully connected layer with 128 neurons.\n",
    "    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])\n",
    "    # Apply sigmoid to layer_1 output for non-linearity.\n",
    "    layer_1 = tf.nn.sigmoid(layer_1)\n",
    "    \n",
    "    # Hidden fully connected layer with 256 neurons.\n",
    "    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])\n",
    "    # Apply sigmoid to layer_2 output for non-linearity.\n",
    "    layer_2 = tf.nn.sigmoid(layer_2)\n",
    "    \n",
    "    # Output fully connected layer with a neuron for each class.\n",
    "    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']\n",
    "    # Apply softmax to normalize the logits to a probability distribution.\n",
    "    return tf.nn.softmax(out_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cross-Entropy loss function.\n",
    "def cross_entropy(y_pred, y_true):\n",
    "    # Encode label to a one hot vector.\n",
    "    y_true = tf.one_hot(y_true, depth=num_classes)\n",
    "    # Clip prediction values to avoid log(0) error.\n",
    "    y_pred = tf.clip_by_value(y_pred, 1e-9, 1.)\n",
    "    # Compute cross-entropy.\n",
    "    return tf.reduce_mean(-tf.reduce_sum(y_true * tf.math.log(y_pred)))\n",
    "\n",
    "# Accuracy metric.\n",
    "def accuracy(y_pred, y_true):\n",
    "    # Predicted class is the index of highest score in prediction vector (i.e. argmax).\n",
    "    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64))\n",
    "    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1)\n",
    "\n",
    "# Stochastic gradient descent optimizer.\n",
    "optimizer = tf.optimizers.SGD(learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimization process. \n",
    "def run_optimization(x, y):\n",
    "    # Wrap computation inside a GradientTape for automatic differentiation.\n",
    "    with tf.GradientTape() as g:\n",
    "        pred = neural_net(x)\n",
    "        loss = cross_entropy(pred, y)\n",
    "        \n",
    "    # Variables to update, i.e. trainable variables.\n",
    "    trainable_variables = list(weights.values()) + list(biases.values())\n",
    "\n",
    "    # Compute gradients.\n",
    "    gradients = g.gradient(loss, trainable_variables)\n",
    "    \n",
    "    # Update W and b following gradients.\n",
    "    optimizer.apply_gradients(zip(gradients, trainable_variables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 100, loss: 571.445923, accuracy: 0.222656\n",
      "step: 200, loss: 405.567535, accuracy: 0.488281\n",
      "step: 300, loss: 252.089172, accuracy: 0.660156\n",
      "step: 400, loss: 192.252136, accuracy: 0.792969\n",
      "step: 500, loss: 129.173553, accuracy: 0.855469\n",
      "step: 600, loss: 125.191071, accuracy: 0.859375\n",
      "step: 700, loss: 103.346634, accuracy: 0.890625\n",
      "step: 800, loss: 120.199402, accuracy: 0.871094\n",
      "step: 900, loss: 95.674088, accuracy: 0.890625\n",
      "step: 1000, loss: 113.775406, accuracy: 0.878906\n",
      "step: 1100, loss: 68.457413, accuracy: 0.941406\n",
      "step: 1200, loss: 80.773163, accuracy: 0.914062\n",
      "step: 1300, loss: 85.862785, accuracy: 0.902344\n",
      "step: 1400, loss: 63.480415, accuracy: 0.949219\n",
      "step: 1500, loss: 77.139435, accuracy: 0.910156\n",
      "step: 1600, loss: 88.129692, accuracy: 0.933594\n",
      "step: 1700, loss: 92.199730, accuracy: 0.906250\n",
      "step: 1800, loss: 90.150421, accuracy: 0.886719\n",
      "step: 1900, loss: 48.567772, accuracy: 0.949219\n",
      "step: 2000, loss: 54.002838, accuracy: 0.941406\n",
      "step: 2100, loss: 58.536209, accuracy: 0.933594\n",
      "step: 2200, loss: 47.156784, accuracy: 0.949219\n",
      "step: 2300, loss: 55.344498, accuracy: 0.949219\n",
      "step: 2400, loss: 70.956612, accuracy: 0.925781\n",
      "step: 2500, loss: 76.179062, accuracy: 0.917969\n",
      "step: 2600, loss: 44.956696, accuracy: 0.929688\n",
      "step: 2700, loss: 56.581280, accuracy: 0.941406\n",
      "step: 2800, loss: 57.775612, accuracy: 0.937500\n",
      "step: 2900, loss: 46.005424, accuracy: 0.960938\n",
      "step: 3000, loss: 51.832504, accuracy: 0.953125\n"
     ]
    }
   ],
   "source": [
    "# Run training for the given number of steps.\n",
    "for step, (batch_x, batch_y) in enumerate(train_data.take(training_steps), 1):\n",
    "    # Run the optimization to update W and b values.\n",
    "    run_optimization(batch_x, batch_y)\n",
    "    \n",
    "    if step % display_step == 0:\n",
    "        pred = neural_net(batch_x)\n",
    "        loss = cross_entropy(pred, batch_y)\n",
    "        acc = accuracy(pred, batch_y)\n",
    "        print(\"step: %i, loss: %f, accuracy: %f\" % (step, loss, acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.937600\n"
     ]
    }
   ],
   "source": [
    "# Test model on validation set.\n",
    "pred = neural_net(x_test)\n",
    "print(\"Test Accuracy: %f\" % accuracy(pred, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize predictions.\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAM20lEQVR4nO3dXahc9bnH8d/vpCmI6UXiS9ik0bTBC8tBEo1BSCxbQktOvIjFIM1FyYHi7kWUFkuo2It4WaQv1JvALkrTkmMJpGoQscmJxVDU4o5Es2NIjCGaxLxYIjQRJMY+vdjLso0za8ZZa2ZN8nw/sJmZ9cya9bDMz7VmvczfESEAV77/aroBAINB2IEkCDuQBGEHkiDsQBJfGeTCbHPoH+iziHCr6ZW27LZX2j5o+7Dth6t8FoD+cq/n2W3PkHRI0nckHZf0mqS1EfFWyTxs2YE+68eWfamkwxFxJCIuSPqTpNUVPg9AH1UJ+zxJx6a9Pl5M+xzbY7YnbE9UWBaAivp+gC4ixiWNS+zGA02qsmU/IWn+tNdfL6YBGEJVwv6apJtsf8P2VyV9X9L2etoCULeed+Mj4qLtByT9RdIMSU9GxP7aOgNQq55PvfW0ML6zA33Xl4tqAFw+CDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9Dw+uyTZPirpnKRPJV2MiCV1NAWgfpXCXrgrIv5Rw+cA6CN244EkqoY9JO2wvcf2WKs32B6zPWF7ouKyAFTgiOh9ZnteRJywfb2knZIejIjdJe/vfWEAuhIRbjW90pY9Ik4Uj2ckPS1paZXPA9A/PYfd9tW2v/bZc0nflTRZV2MA6lXlaPxcSU/b/uxz/i8iXqilKwC1q/Sd/UsvjO/sQN/15Ts7gMsHYQeSIOxAEoQdSIKwA0nUcSNMCmvWrGlbu//++0vnff/990vrH3/8cWl9y5YtpfVTp061rR0+fLh0XuTBlh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkuCuty4dOXKkbW3BggWDa6SFc+fOta3t379/gJ0Ml+PHj7etPfbYY6XzTkxcvr+ixl1vQHKEHUiCsANJEHYgCcIOJEHYgSQIO5AE97N3qeye9VtuuaV03gMHDpTWb7755tL6rbfeWlofHR1tW7vjjjtK5z127Fhpff78+aX1Ki5evFha/+CDD0rrIyMjPS/7vffeK61fzufZ22HLDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJcD/7FWD27Nlta4sWLSqdd8+ePaX122+/vZeWutLp9/IPHTpUWu90/cKcOXPa1tavX18676ZNm0rrw6zn+9ltP2n7jO3JadPm2N5p++3isf2/NgBDoZvd+N9LWnnJtIcl7YqImyTtKl4DGGIdwx4RuyWdvWTyakmbi+ebJd1Tb1sA6tbrtfFzI+Jk8fyUpLnt3mh7TNJYj8sBUJPKN8JERJQdeIuIcUnjEgfogCb1eurttO0RSSoez9TXEoB+6DXs2yWtK56vk/RsPe0A6JeO59ltPyVpVNK1kk5L2ijpGUlbJd0g6V1J90XEpQfxWn0Wu/Ho2r333lta37p1a2l9cnKybe2uu+4qnffs2Y7/nIdWu/PsHb+zR8TaNqUVlToCMFBcLgskQdiBJAg7kARhB5Ig7EAS3OKKxlx//fWl9X379lWaf82aNW1r27ZtK533csaQzUByhB1IgrADSRB2IAnCDiRB2IEkCDuQBEM2ozGdfs75uuuuK61/+OGHpfWDBw9+6Z6uZGzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJ7mdHXy1btqxt7cUXXyydd+bMmaX10dHR0vru3btL61cq7mcHkiPsQBKEHUiCsANJEHYgCcIOJEHYgSS4nx19tWrVqra1TufRd+3aVVp/5ZVXeuopq45bdttP2j5je3LatEdtn7C9t/hr/18UwFDoZjf+95JWtpj+m4hYVPw9X29bAOrWMewRsVvS2QH0AqCPqhyge8D2m8Vu/ux2b7I9ZnvC9kSFZQGoqNewb5K0UNIiSScl/ardGyNiPCKWRMSSHpcFoAY9hT0iTkfEpxHxL0m/k7S03rYA1K2nsNsemfbye5Im270XwHDoeJ7d9lOSRiVda/u4pI2SRm0vkhSSjkr6Uf9axDC76qqrSusrV7Y6kTPlwoULpfNu3LixtP7JJ5+U1vF5HcMeEWtbTH6iD70A6CMulwWSIOxAEoQdSIKwA0kQdiAJbnFFJRs2bCitL168uG3thRdeKJ335Zdf7qkntMaWHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYMhmlLr77rtL688880xp/aOPPmpbK7v9VZJeffXV0jpaY8hmIDnCDiRB2IEkCDuQBGEHkiDsQBKEHUiC+9mTu+aaa0rrjz/+eGl9xowZpfXnn28/5ifn0QeLLTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJMH97Fe4TufBO53rvu2220rr77zzTmm97J71TvOiNz3fz257vu2/2n7L9n7bPy6mz7G90/bbxePsupsGUJ9uduMvSvppRHxL0h2S1tv+lqSHJe2KiJsk7SpeAxhSHcMeEScj4vXi+TlJByTNk7Ra0ubibZsl3dOnHgHU4EtdG297gaTFkv4uaW5EnCxKpyTNbTPPmKSxCj0CqEHXR+Ntz5K0TdJPIuKf02sxdZSv5cG3iBiPiCURsaRSpwAq6SrstmdqKuhbIuLPxeTTtkeK+oikM/1pEUAdOu7G27akJyQdiIhfTyttl7RO0i+Kx2f70iEqWbhwYWm906m1Th566KHSOqfXhkc339mXSfqBpH229xbTHtFUyLfa/qGkdyXd15cOAdSiY9gj4m+SWp6kl7Si3nYA9AuXywJJEHYgCcIOJEHYgSQIO5AEPyV9Bbjxxhvb1nbs2FHpszds2FBaf+655yp9PgaHLTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJMF59ivA2Fj7X/264YYbKn32Sy+9VFof5E+Roxq27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOfZLwPLly8vrT/44IMD6gSXM7bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BEN+Ozz5f0B0lzJYWk8Yj4re1HJd0v6YPirY9ExPP9ajSzO++8s7Q+a9asnj+70/jp58+f7/mzMVy6uajmoqSfRsTrtr8maY/tnUXtNxHxy/61B6Au3YzPflLSyeL5OdsHJM3rd2MA6vWlvrPbXiBpsaS/F5MesP2m7Sdtz24zz5jtCdsT1VoFUEXXYbc9S9I2ST+JiH9K2iRpoaRFmtry/6rVfBExHhFLImJJ9XYB9KqrsNueqamgb4mIP0tSRJyOiE8j4l+Sfidpaf/aBFBVx7DbtqQnJB2IiF9Pmz4y7W3fkzRZf3sA6tLN0fhlkn4gaZ/tvcW0RySttb1IU6fjjkr6UR/6Q0VvvPFGaX3FihWl9bNnz9bZDhrUzdH4v0lyixLn1IHLCFfQAUkQdiAJwg4kQdiBJAg7kARhB5LwIIfctc34vkCfRUSrU+Vs2YEsCDuQBGEHkiDsQBKEHUiCsANJEHYgiUEP2fwPSe9Oe31tMW0YDWtvw9qXRG+9qrO3G9sVBnpRzRcWbk8M62/TDWtvw9qXRG+9GlRv7MYDSRB2IImmwz7e8PLLDGtvw9qXRG+9GkhvjX5nBzA4TW/ZAQwIYQeSaCTstlfaPmj7sO2Hm+ihHdtHbe+zvbfp8emKMfTO2J6cNm2O7Z223y4eW46x11Bvj9o+Uay7vbZXNdTbfNt/tf2W7f22f1xMb3TdlfQ1kPU28O/stmdIOiTpO5KOS3pN0tqIeGugjbRh+6ikJRHR+AUYtr8t6bykP0TEfxfTHpN0NiJ+UfyPcnZE/GxIentU0vmmh/EuRisamT7MuKR7JP2vGlx3JX3dpwGstya27EslHY6IIxFxQdKfJK1uoI+hFxG7JV06JMtqSZuL55s19Y9l4Nr0NhQi4mREvF48Pyfps2HGG113JX0NRBNhnyfp2LTXxzVc472HpB2299gea7qZFuZGxMni+SlJc5tspoWOw3gP0iXDjA/Nuutl+POqOED3Rcsj4lZJ/yNpfbG7OpRi6jvYMJ077WoY70FpMcz4fzS57nod/ryqJsJ+QtL8aa+/XkwbChFxong8I+lpDd9Q1Kc/G0G3eDzTcD//MUzDeLcaZlxDsO6aHP68ibC/Jukm29+w/VVJ35e0vYE+vsD21cWBE9m+WtJ3NXxDUW+XtK54vk7Ssw328jnDMox3u2HG1fC6a3z484gY+J+kVZo6Iv+OpJ830UObvr4p6Y3ib3/TvUl6SlO7dZ9o6tjGDyVdI2mXpLcl/b+kOUPU2x8l7ZP0pqaCNdJQb8s1tYv+pqS9xd+qptddSV8DWW9cLgskwQE6IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUji3y9hG/l2EQpSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model prediction: 7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAANYElEQVR4nO3df6hc9ZnH8c9n3QTEFk0ie7kYWWvUP+KiVq6yuLK41EZXNDEgNUEWS4X0jwoV44+QFSIsouxud/8MpDQ0atemITGNddnUDfXHggleJcZE02oksQk3CdmATRCpSZ79454st3rnzM05Z+ZM8rxfcJmZ88yc8zD6yfk153wdEQJw7vuzthsA0B+EHUiCsANJEHYgCcIOJPHn/VyYbQ79Az0WEZ5seq01u+3bbf/W9ke2l9WZF4DectXz7LbPk/Q7Sd+WtF/SW5IWR8T7JZ9hzQ70WC/W7DdK+igiPo6IP0r6uaQFNeYHoIfqhP0SSb+f8Hp/Me1P2F5ie9T2aI1lAaip5wfoImKVpFUSm/FAm+qs2Q9IunTC69nFNAADqE7Y35J0pe1v2J4uaZGkTc20BaBplTfjI+KE7QclbZZ0nqTVEbGrsc4ANKryqbdKC2OfHei5nvyoBsDZg7ADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgST6eitpVPPII4+U1s8///yOtWuuuab0s/fcc0+lnk5buXJlaf3NN9/sWHvuuedqLRtnhjU7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTB3WUHwNq1a0vrdc+Ft2nPnj0da7feemvpZz/55JOm20mBu8sCyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJcz94HbZ5H3717d2l98+bNpfXLL7+8tH7XXXeV1ufMmdOxdt9995V+9umnny6t48zUCrvtvZKOSTop6UREjDTRFIDmNbFm/7uIONLAfAD0EPvsQBJ1wx6Sfm37bdtLJnuD7SW2R22P1lwWgBrqbsbfHBEHbP+FpFds746I1ye+ISJWSVolcSEM0KZaa/aIOFA8Hpb0oqQbm2gKQPMqh932Bba/fvq5pHmSdjbVGIBm1dmMH5L0ou3T8/mPiPivRro6y4yMlJ9xXLhwYa3579q1q7Q+f/78jrUjR8pPlBw/fry0Pn369NL61q1bS+vXXnttx9qsWbNKP4tmVQ57RHwsqfN/SQADhVNvQBKEHUiCsANJEHYgCcIOJMElrg0YHh4urRenJzvqdmrttttuK62PjY2V1utYunRpaX3u3LmV5/3yyy9X/izOHGt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiC8+wNeOmll0rrV1xxRWn92LFjpfWjR4+ecU9NWbRoUWl92rRpfeoEdbFmB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkOM/eB/v27Wu7hY4effTR0vpVV11Va/7btm2rVEPzWLMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKOiP4tzO7fwiBJuvPOO0vr69atK613G7L58OHDpfWy6+Ffe+210s+imoiYdKCCrmt226ttH7a9c8K0mbZfsf1h8TijyWYBNG8qm/E/lXT7l6Ytk7QlIq6UtKV4DWCAdQ17RLwu6cv3RVogaU3xfI2ku5ttC0DTqv42figiTg8wdlDSUKc32l4iaUnF5QBoSO0LYSIiyg68RcQqSaskDtABbap66u2Q7WFJKh7LD8kCaF3VsG+SdH/x/H5Jv2ymHQC90nUz3vYLkm6RdLHt/ZJWSHpG0i9sPyBpn6Tv9LJJVDcyMlJa73YevZu1a9eW1jmXPji6hj0iFncofavhXgD0ED+XBZIg7EAShB1IgrADSRB2IAluJX0O2LhxY8favHnzas372WefLa0/8cQTteaP/mHNDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJcCvps8Dw8HBp/d133+1YmzVrVulnjxw5Ulq/6aabSut79uwpraP/Kt9KGsC5gbADSRB2IAnCDiRB2IEkCDuQBGEHkuB69rPA+vXrS+vdzqWXef7550vrnEc/d7BmB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkOM8+AObPn19av/766yvP+9VXXy2tr1ixovK8cXbpuma3vdr2Yds7J0x70vYB29uLvzt62yaAuqayGf9TSbdPMv3fI+K64u8/m20LQNO6hj0iXpd0tA+9AOihOgfoHrS9o9jMn9HpTbaX2B61PVpjWQBqqhr2lZLmSLpO0pikH3V6Y0SsioiRiBipuCwADagU9og4FBEnI+KUpB9LurHZtgA0rVLYbU+8t/FCSTs7vRfAYOh6nt32C5JukXSx7f2SVki6xfZ1kkLSXknf712LZ79u15svX768tD5t2rTKy96+fXtp/fjx45XnjbNL17BHxOJJJv+kB70A6CF+LgskQdiBJAg7kARhB5Ig7EASXOLaB0uXLi2t33DDDbXmv3Hjxo41LmHFaazZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJR0T/Fmb3b2ED5PPPPy+t17mEVZJmz57dsTY2NlZr3jj7RIQnm86aHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4Hr2c8DMmTM71r744os+dvJVn376acdat966/f7gwgsvrNSTJF100UWl9YcffrjyvKfi5MmTHWuPP/546Wc/++yzSstkzQ4kQdiBJAg7kARhB5Ig7EAShB1IgrADSXCe/RywY8eOtlvoaN26dR1r3a61HxoaKq3fe++9lXoadAcPHiytP/XUU5Xm23XNbvtS27+x/b7tXbZ/WEyfafsV2x8WjzMqdQCgL6ayGX9C0tKImCvpryX9wPZcScskbYmIKyVtKV4DGFBdwx4RYxHxTvH8mKQPJF0iaYGkNcXb1ki6u0c9AmjAGe2z275M0jclbZM0FBGnd7oOSpp0B8v2EklLavQIoAFTPhpv+2uS1kt6KCL+MLEW43etnPRmkhGxKiJGImKkVqcAaplS2G1P03jQfxYRG4rJh2wPF/VhSYd70yKAJnS9lbRta3yf/GhEPDRh+r9I+t+IeMb2MkkzI+KxLvNKeSvpDRs2lNYXLFjQp05yOXHiRMfaqVOnas1706ZNpfXR0dHK837jjTdK61u3bi2td7qV9FT22f9G0j9Ies/29mLacknPSPqF7Qck7ZP0nSnMC0BLuoY9Iv5H0qT/Ukj6VrPtAOgVfi4LJEHYgSQIO5AEYQeSIOxAEgzZPAAee6z05wm1h3Quc/XVV5fWe3kZ6erVq0vre/furTX/9evXd6zt3r271rwHGUM2A8kRdiAJwg4kQdiBJAg7kARhB5Ig7EASnGcHzjGcZweSI+xAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkuobd9qW2f2P7fdu7bP+wmP6k7QO2txd/d/S+XQBVdb15he1hScMR8Y7tr0t6W9LdGh+P/XhE/OuUF8bNK4Ce63TziqmMzz4maax4fsz2B5IuabY9AL12Rvvsti+T9E1J24pJD9reYXu17RkdPrPE9qjt0XqtAqhjyvegs/01Sa9JeioiNtgeknREUkj6J41v6n+vyzzYjAd6rNNm/JTCbnuapF9J2hwR/zZJ/TJJv4qIv+oyH8IO9FjlG07atqSfSPpgYtCLA3enLZS0s26TAHpnKkfjb5b0hqT3JJ0qJi+XtFjSdRrfjN8r6fvFwbyyebFmB3qs1mZ8Uwg70HvcNx5IjrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5BE1xtONuyIpH0TXl9cTBtEg9rboPYl0VtVTfb2l50Kfb2e/SsLt0cjYqS1BkoMam+D2pdEb1X1qzc244EkCDuQRNthX9Xy8ssMam+D2pdEb1X1pbdW99kB9E/ba3YAfULYgSRaCbvt223/1vZHtpe10UMntvfafq8YhrrV8emKMfQO2945YdpM26/Y/rB4nHSMvZZ6G4hhvEuGGW/1u2t7+PO+77PbPk/S7yR9W9J+SW9JWhwR7/e1kQ5s75U0EhGt/wDD9t9KOi7p2dNDa9n+Z0lHI+KZ4h/KGRHx+ID09qTOcBjvHvXWaZjx76rF767J4c+raGPNfqOkjyLi44j4o6SfS1rQQh8DLyJel3T0S5MXSFpTPF+j8f9Z+q5DbwMhIsYi4p3i+TFJp4cZb/W7K+mrL9oI+yWSfj/h9X4N1njvIenXtt+2vaTtZiYxNGGYrYOShtpsZhJdh/Hupy8NMz4w312V4c/r4gDdV90cEddL+ntJPyg2VwdSjO+DDdK505WS5mh8DMAxST9qs5limPH1kh6KiD9MrLX53U3SV1++tzbCfkDSpRNezy6mDYSIOFA8Hpb0osZ3OwbJodMj6BaPh1vu5/9FxKGIOBkRpyT9WC1+d8Uw4+sl/SwiNhSTW//uJuurX99bG2F/S9KVtr9he7qkRZI2tdDHV9i+oDhwItsXSJqnwRuKepOk+4vn90v6ZYu9/IlBGca70zDjavm7a33484jo+5+kOzR+RH6PpH9so4cOfV0u6d3ib1fbvUl6QeObdV9o/NjGA5JmSdoi6UNJ/y1p5gD19pzGh/beofFgDbfU280a30TfIWl78XdH299dSV99+d74uSyQBAfogCQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJ/wN8jzcem5JvKwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model prediction: 2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAMEElEQVR4nO3dXYhc5R3H8d+vabwwepFUE4OKsRJRUUzKIoKhWnzBBiHmRoxQEiqsFwYi9KJiLxRKQaTaCy+EFcU0WF+IBqPWaBrEtDeaVVNNfIlWIiasWSWCb4g1+fdiT8oad85s5pwzZ9z/9wPLzDzPnDl/DvnlOXNe5nFECMDM95O2CwDQH4QdSIKwA0kQdiAJwg4k8dN+rsw2h/6BhkWEp2qvNLLbvtr2u7bft31rlc8C0Cz3ep7d9ixJeyRdKWmfpB2SVkXEWyXLMLIDDWtiZL9I0vsR8UFEfCvpUUkrKnwegAZVCfupkj6a9Hpf0fY9todtj9oerbAuABU1foAuIkYkjUjsxgNtqjKy75d0+qTXpxVtAAZQlbDvkLTY9pm2j5N0vaTN9ZQFoG4978ZHxHe210p6XtIsSQ9GxO7aKgNQq55PvfW0Mr6zA41r5KIaAD8ehB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4k0dcpm5HP2Wef3bHvnXfeKV123bp1pf333ntvTzVlxcgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwnh2NWrp0ace+w4cPly67b9++ustJrVLYbe+V9IWkQ5K+i4ihOooCUL86RvZfRcSnNXwOgAbxnR1IomrYQ9ILtl+1PTzVG2wP2x61PVpxXQAqqLobvywi9tueL2mr7XciYvvkN0TEiKQRSbIdFdcHoEeVRvaI2F88jkvaJOmiOooCUL+ew257ju0TjzyXdJWkXXUVBqBeVXbjF0jaZPvI5/wtIrbUUhVmjCVLlnTs++qrr0qX3bRpU83V5NZz2CPiA0kX1lgLgAZx6g1IgrADSRB2IAnCDiRB2IEkuMUVlZx//vml/WvXru3Yt2HDhrrLQQlGdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgvPsqOScc84p7Z8zZ07Hvscee6zuclCCkR1IgrADSRB2IAnCDiRB2IEkCDuQBGEHknBE/yZpYUaYmeeVV14p7T/55JM79nW7F77bT01jahHhqdoZ2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCe5nR6lFixaV9g8NDZX279mzp2Mf59H7q+vIbvtB2+O2d01qm2d7q+33ise5zZYJoKrp7MY/JOnqo9pulbQtIhZL2la8BjDAuoY9IrZLOnhU8wpJ64vn6yVdW29ZAOrW63f2BRExVjz/WNKCTm+0PSxpuMf1AKhJ5QN0ERFlN7hExIikEYkbYYA29Xrq7YDthZJUPI7XVxKAJvQa9s2SVhfPV0t6qp5yADSl62687UckXSbpJNv7JN0u6U5Jj9u+UdKHkq5rski059JLL620/CeffFJTJaiqa9gjYlWHrstrrgVAg7hcFkiCsANJEHYgCcIOJEHYgSS4xRWlLrjggkrL33XXXTVVgqoY2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCaZsTu7iiy8u7X/22WdL+/fu3Vvaf8kll3Ts++abb0qXRW+YshlIjrADSRB2IAnCDiRB2IEkCDuQBGEHkuB+9uSuuOKK0v558+aV9m/ZsqW0n3Ppg4ORHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4Dx7chdeeGFpf7ffO9i4cWOd5aBBXUd22w/aHre9a1LbHbb3295Z/C1vtkwAVU1nN/4hSVdP0f6XiFhS/P293rIA1K1r2CNiu6SDfagFQIOqHKBba/uNYjd/bqc32R62PWp7tMK6AFTUa9jvk3SWpCWSxiTd3emNETESEUMRMdTjugDUoKewR8SBiDgUEYcl3S/ponrLAlC3nsJue+Gklysl7er0XgCDoevvxtt+RNJlkk6SdEDS7cXrJZJC0l5JN0XEWNeV8bvxfXfKKaeU9u/cubO0/7PPPivtP/fcc4+1JDSs0+/Gd72oJiJWTdH8QOWKAPQVl8sCSRB2IAnCDiRB2IEkCDuQBLe4znBr1qwp7Z8/f35p/3PPPVdjNWgTIzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJMF59hnujDPOqLR8t1tc8ePByA4kQdiBJAg7kARhB5Ig7EAShB1IgrADSXCefYa75pprKi3/9NNP11QJ2sbIDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJcJ59Bli2bFnHvm5TNiOPriO77dNtv2j7Ldu7ba8r2ufZ3mr7veJxbvPlAujVdHbjv5P0u4g4T9LFkm62fZ6kWyVti4jFkrYVrwEMqK5hj4ixiHiteP6FpLclnSpphaT1xdvWS7q2oRoB1OCYvrPbXiRpqaSXJS2IiLGi62NJCzosMyxpuEKNAGow7aPxtk+Q9ISkWyLi88l9ERGSYqrlImIkIoYiYqhSpQAqmVbYbc/WRNAfjogni+YDthcW/QsljTdTIoA6dN2Nt21JD0h6OyLumdS1WdJqSXcWj081UiG6WrlyZce+WbNmlS77+uuvl/Zv3769p5oweKbznf0SSb+R9KbtnUXbbZoI+eO2b5T0oaTrGqkQQC26hj0i/iXJHbovr7ccAE3hclkgCcIOJEHYgSQIO5AEYQeS4BbXH4Hjjz++tH/58uU9f/bGjRtL+w8dOtTzZ2OwMLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKe+JGZPq3M7t/KZpDZs2eX9r/00ksd+8bHy39T5IYbbijt//rrr0v7MXgiYsq7VBnZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJzrMDMwzn2YHkCDuQBGEHkiDsQBKEHUiCsANJEHYgia5ht3267Rdtv2V7t+11Rfsdtvfb3ln89f7j5QAa1/WiGtsLJS2MiNdsnyjpVUnXamI+9i8j4s/TXhkX1QCN63RRzXTmZx+TNFY8/8L225JOrbc8AE07pu/sthdJWirp5aJpre03bD9oe26HZYZtj9oerVYqgCqmfW287RMkvSTpTxHxpO0Fkj6VFJL+qIld/d92+Qx244GGddqNn1bYbc+W9Iyk5yPinin6F0l6JiLO7/I5hB1oWM83wti2pAckvT056MWBuyNWStpVtUgAzZnO0fhlkv4p6U1Jh4vm2yStkrREE7vxeyXdVBzMK/ssRnagYZV24+tC2IHmcT87kBxhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgia4/OFmzTyV9OOn1SUXbIBrU2ga1LonaelVnbWd06ujr/ew/WLk9GhFDrRVQYlBrG9S6JGrrVb9qYzceSIKwA0m0HfaRltdfZlBrG9S6JGrrVV9qa/U7O4D+aXtkB9AnhB1IopWw277a9ru237d9axs1dGJ7r+03i2moW52frphDb9z2rklt82xvtf1e8TjlHHst1TYQ03iXTDPe6rZre/rzvn9ntz1L0h5JV0raJ2mHpFUR8VZfC+nA9l5JQxHR+gUYtn8p6UtJfz0ytZbtuyQdjIg7i/8o50bE7wektjt0jNN4N1Rbp2nG16jFbVfn9Oe9aGNkv0jS+xHxQUR8K+lRSStaqGPgRcR2SQePal4haX3xfL0m/rH0XYfaBkJEjEXEa8XzLyQdmWa81W1XUldftBH2UyV9NOn1Pg3WfO8h6QXbr9oebruYKSyYNM3Wx5IWtFnMFLpO491PR00zPjDbrpfpz6viAN0PLYuIX0j6taSbi93VgRQT38EG6dzpfZLO0sQcgGOS7m6zmGKa8Sck3RIRn0/ua3PbTVFXX7ZbG2HfL+n0Sa9PK9oGQkTsLx7HJW3SxNeOQXLgyAy6xeN4y/X8X0QciIhDEXFY0v1qcdsV04w/IenhiHiyaG59201VV7+2Wxth3yFpse0zbR8n6XpJm1uo4wdszykOnMj2HElXafCmot4saXXxfLWkp1qs5XsGZRrvTtOMq+Vt1/r05xHR9z9JyzVxRP4/kv7QRg0d6vq5pH8Xf7vbrk3SI5rYrfuvJo5t3CjpZ5K2SXpP0j8kzRug2jZoYmrvNzQRrIUt1bZME7vob0jaWfwtb3vbldTVl+3G5bJAEhygA5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk/gciQMnFdlEPHAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model prediction: 1\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAANrUlEQVR4nO3df4gU9xnH8c+jbf+x/UPrVcyPaluDQQqNxZhCg0lTWjQQvP6RRgnBksKZYKKBQisKqaEUQtKm/0SUCwm9ljalYNIeIq2pSG1ASs6QH+aubX6gVrmcMUIakRCjT//YMZx6853LzszOns/7BcfuzrM7+2SST2Z2vzvzNXcXgMvftKYbANAZhB0IgrADQRB2IAjCDgTxqU6+mZnx1T9QM3e3iZaX2rOb2XIz+7eZvWFmG8usC0C9rN1xdjObLuk/kr4j6aikFyStdvfhxGvYswM1q2PPvlTSG+7+lrt/KOkPklaWWB+AGpUJ+5WS/jvu8dFs2QXMrM/MhsxsqMR7ASip9i/o3L1fUr/EYTzQpDJ79mOSrh73+KpsGYAuVCbsL0i6xsy+ZGafkbRK0mA1bQGoWtuH8e7+kZndJ+mvkqZLesrdX6usMwCVanvora034zM7ULtaflQDYOog7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiCIjk7ZjM6bMWNGsv7oo48m62vXrk3WDxw4kKzffvvtubXDhw8nX4tqsWcHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSCYxfUyt2DBgmR9ZGSk1PqnTUvvL9avX59b27p1a6n3xsTyZnEt9aMaMzsk6X1JZyV95O5LyqwPQH2q+AXdt9z9RAXrAVAjPrMDQZQNu0vabWYHzKxvoieYWZ+ZDZnZUMn3AlBC2cP4G939mJl9QdJzZvYvd983/gnu3i+pX+ILOqBJpfbs7n4suz0u6VlJS6toCkD12g67mc0ws8+dvy/pu5IOVtUYgGqVOYyfI+lZMzu/nt+7+18q6QqfSE9PT25tYGCgg52gm7Uddnd/S9LXKuwFQI0YegOCIOxAEIQdCIKwA0EQdiAILiU9BaROE5Wk3t7e3NrSpc3+zmnZsmW5taLTY19++eVkfd++fck6LsSeHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeC4FLSU8DZs2eT9XPnznWok0sVjZWX6a1oSuc77rgjWS+aTvpylXcpafbsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAE4+xdYNeuXcn6ihUrkvUmx9nffffdZP3UqVO5tXnz5lXdzgWmT59e6/q7FePsQHCEHQiCsANBEHYgCMIOBEHYgSAIOxAE143vgJtuuilZX7hwYbJeNI5e5zj79u3bk/Xdu3cn6++9915u7ZZbbkm+dvPmzcl6kXvvvTe3tm3btlLrnooK9+xm9pSZHTezg+OWzTKz58zs9ex2Zr1tAihrMofxv5a0/KJlGyXtcfdrJO3JHgPoYoVhd/d9kk5etHilpIHs/oCk3mrbAlC1dj+zz3H30ez+25Lm5D3RzPok9bX5PgAqUvoLOnf31Aku7t4vqV/iRBigSe0OvY2Z2VxJym6PV9cSgDq0G/ZBSWuy+2sk/bmadgDUpfB8djN7WtLNkmZLGpP0U0l/kvRHSV+UdFjS99394i/xJlrXZXkYP3/+/GR9//79yfrs2bOT9TLXZi+69vqOHTuS9YceeihZP336dLKeUnQ+e9F26+npSdY/+OCD3NqDDz6YfO3jjz+erJ85cyZZb1Le+eyFn9ndfXVO6dulOgLQUfxcFgiCsANBEHYgCMIOBEHYgSC4lHQFFixYkKyPjIyUWn/R0NvevXtza6tWrUq+9sSJE2311An3339/sv7YY48l66ntVnRa8LXXXpusv/nmm8l6k7iUNBAcYQeCIOxAEIQdCIKwA0EQdiAIwg4EwaWkp4ChoaFk/e67786tdfM4epHBwcFk/c4770zWr7/++irbmfLYswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyzd0DR+ehFbrjhhoo6mVrMJjwt+2NF27XMdt+yZUuyftddd7W97qawZweCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIBhnr8A999yTrBddoxwTu+2225L1xYsXJ+up7V7076RonH0qKtyzm9lTZnbczA6OW7bFzI6Z2UvZ3631tgmgrMkcxv9a0vIJlv/K3a/L/nZV2xaAqhWG3d33STrZgV4A1KjMF3T3mdkr2WH+zLwnmVmfmQ2ZWfpCagBq1W7Yt0n6iqTrJI1K+mXeE929392XuPuSNt8LQAXaCru7j7n7WXc/J+kJSUurbQtA1doKu5nNHffwe5IO5j0XQHcoHGc3s6cl3SxptpkdlfRTSTeb2XWSXNIhSWvra7H7FY0HR9bT05NbW7RoUfK1mzZtqrqdj73zzjvJ+pkzZ2p776YUht3dV0+w+MkaegFQI34uCwRB2IEgCDsQBGEHgiDsQBCc4opabd68Obe2bt26Wt/70KFDubU1a9YkX3vkyJGKu2kee3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIJxdpSya1f6WqMLFy7sUCeXGh4ezq09//zzHeykO7BnB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgGGevgJkl69Omlft/6ooVK9p+bX9/f7J+xRVXtL1uqfifrcnpqrnE94XYswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyzV2Dbtm3J+iOPPFJq/Tt37kzWy4xl1z0OXuf6t2/fXtu6L0eFe3Yzu9rM9prZsJm9ZmYbsuWzzOw5M3s9u51Zf7sA2jWZw/iPJP3I3RdJ+oakdWa2SNJGSXvc/RpJe7LHALpUYdjdfdTdX8zuvy9pRNKVklZKGsieNiCpt6YeAVTgE31mN7P5khZL+qekOe4+mpXeljQn5zV9kvpK9AigApP+Nt7MPitph6QH3P1/42vu7pJ8ote5e7+7L3H3JaU6BVDKpMJuZp9WK+i/c/dnssVjZjY3q8+VdLyeFgFUwVo75cQTWudvDkg66e4PjFv+qKR33f1hM9soaZa7/7hgXek3m6LmzZuXrO/fvz9Z7+npSda7+TTSot7GxsZyayMjI8nX9vWlP/2Njo4m66dPn07WL1fuPuE515P5zP5NSXdJetXMXsqWbZL0sKQ/mtkPJR2W9P0K+gRQk8Kwu/vzkvKuzvDtatsBUBd+LgsEQdiBIAg7EARhB4Ig7EAQhePslb7ZZTrOXmTZsmXJem9vb7K+YcOGZL2bx9nXr1+fW9u6dWvV7UD54+zs2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMbZp4Dly5cn66nzvoumLR4cHEzWi6Z8Lpquenh4OLd25MiR5GvRHsbZgeAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtmBywzj7EBwhB0IgrADQRB2IAjCDgRB2IEgCDsQRGHYzexqM9trZsNm9pqZbciWbzGzY2b2UvZ3a/3tAmhX4Y9qzGyupLnu/qKZfU7SAUm9as3HfsrdfzHpN+NHNUDt8n5UM5n52UcljWb33zezEUlXVtsegLp9os/sZjZf0mJJ/8wW3Wdmr5jZU2Y2M+c1fWY2ZGZD5VoFUMakfxtvZp+V9HdJP3f3Z8xsjqQTklzSz9Q61L+7YB0cxgM1yzuMn1TYzezTknZK+qu7PzZBfb6kne7+1YL1EHagZm2fCGOty4c+KWlkfNCzL+7O+56kg2WbBFCfyXwbf6Okf0h6VdL5uYE3SVot6Tq1DuMPSVqbfZmXWhd7dqBmpQ7jq0LYgfpxPjsQHGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiCIwgtOVuyEpMPjHs/OlnWjbu2tW/uS6K1dVfY2L6/Q0fPZL3lzsyF3X9JYAwnd2lu39iXRW7s61RuH8UAQhB0Ioumw9zf8/ind2lu39iXRW7s60lujn9kBdE7Te3YAHULYgSAaCbuZLTezf5vZG2a2sYke8pjZITN7NZuGutH56bI59I6b2cFxy2aZ2XNm9np2O+Ecew311hXTeCemGW902zU9/XnHP7Ob2XRJ/5H0HUlHJb0gabW7D3e0kRxmdkjSEndv/AcYZrZM0ilJvzk/tZaZPSLppLs/nP2Pcqa7/6RLetuiTziNd0295U0z/gM1uO2qnP68HU3s2ZdKesPd33L3DyX9QdLKBvroeu6+T9LJixavlDSQ3R9Q6z+WjsvprSu4+6i7v5jdf1/S+WnGG912ib46oomwXynpv+MeH1V3zffuknab2QEz62u6mQnMGTfN1tuS5jTZzAQKp/HupIumGe+abdfO9Odl8QXdpW50969LWiFpXXa42pW89Rmsm8ZOt0n6ilpzAI5K+mWTzWTTjO+Q9IC7/298rcltN0FfHdluTYT9mKSrxz2+KlvWFdz9WHZ7XNKzan3s6CZj52fQzW6PN9zPx9x9zN3Puvs5SU+owW2XTTO+Q9Lv3P2ZbHHj226ivjq13ZoI+wuSrjGzL5nZZyStkjTYQB+XMLMZ2RcnMrMZkr6r7puKelDSmuz+Gkl/brCXC3TLNN5504yr4W3X+PTn7t7xP0m3qvWN/JuSNjfRQ05fX5b0cvb3WtO9SXparcO6M2p9t/FDSZ+XtEfS65L+JmlWF/X2W7Wm9n5FrWDNbai3G9U6RH9F0kvZ361Nb7tEXx3ZbvxcFgiCL+iAIAg7EARhB4Ig7EAQhB0IgrADQRB2IIj/A8nhboC3dEL1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model prediction: 0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAANTUlEQVR4nO3db6hc9Z3H8c9nTRsxDZK7wRDSsKlRkBDcVIMoG1alNGYjEotaEsKSVdnbBxVa3AcrKlTUBZFtln1i4Bal6dJNKRox1LKtDXFdn5TcSFav3m2NIZKEmBhDaCKBavLdB/dErnrnzM3MOXPOzff9gsvMnO+cmS/HfPydPzPzc0QIwMXvL5puAMBgEHYgCcIOJEHYgSQIO5DErEG+mW1O/QM1iwhPtbyvkd32Gtt/sL3P9kP9vBaAernX6+y2L5H0R0nflnRI0m5JGyLinZJ1GNmBmtUxst8gaV9E7I+IP0v6haR1fbwegBr1E/ZFkg5OenyoWPY5todtj9oe7eO9APSp9hN0ETEiaURiNx5oUj8j+2FJiyc9/nqxDEAL9RP23ZKutv0N21+VtF7SjmraAlC1nnfjI+JT2w9I+o2kSyQ9FxFvV9YZgEr1fOmtpzfjmB2oXS0fqgEwcxB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRM9TNmNwrrvuutL69u3bO9aWLFlScTftsXr16tL6+Ph4x9rBgwerbqf1+gq77QOSTkk6K+nTiFhZRVMAqlfFyH5rRByv4HUA1IhjdiCJfsMekn5re4/t4ameYHvY9qjt0T7fC0Af+t2NXxURh21fIekV2/8XEa9NfkJEjEgakSTb0ef7AehRXyN7RBwubo9JelHSDVU0BaB6PYfd9hzbc8/fl7Ra0lhVjQGoVj+78QskvWj7/Ov8Z0T8VyVd4XNuu+220vrs2bMH1Em73HHHHaX1++67r2Nt/fr1VbfTej2HPSL2S/rrCnsBUCMuvQFJEHYgCcIOJEHYgSQIO5AEX3FtgVmzyv8zrF27dkCdzCx79uwprT/44IMda3PmzCld9+OPP+6ppzZjZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJLjO3gK33npraf2mm24qrT/99NNVtjNjzJs3r7S+bNmyjrXLLrusdF2uswOYsQg7kARhB5Ig7EAShB1IgrADSRB2IAlHDG6Slqwzwixfvry0/uqrr5bWP/roo9L69ddf37F2+vTp0nVnsm7bbdWqVR1rCxcuLF33ww8/7KWlVogIT7WckR1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkuD77APw6KOPlta7/Yb5mjVrSusX67X0oaGh0vrNN99cWj937lyV7cx4XUd228/ZPmZ7bNKyIduv2H63uC3/FQEAjZvObvxPJX1xaHlI0s6IuFrSzuIxgBbrGvaIeE3SiS8sXidpa3F/q6Q7q20LQNV6PWZfEBFHivsfSFrQ6Ym2hyUN9/g+ACrS9wm6iIiyL7hExIikESnvF2GANuj10ttR2wslqbg9Vl1LAOrQa9h3SNpU3N8k6aVq2gFQl6678ba3SbpF0nzbhyT9SNJTkn5p+35J70v6bp1Ntt3dd99dWu82v/q+fftK66Ojoxfc08XgkUceKa13u45e9n33kydP9tDRzNY17BGxoUPpWxX3AqBGfFwWSIKwA0kQdiAJwg4kQdiBJPiKawXuueee0nq36YGfeeaZKtuZMZYsWVJa37hxY2n97NmzpfUnn3yyY+2TTz4pXfdixMgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwnX2aLr/88o61G2+8sa/X3rJlS1/rz1TDw+W/VjZ//vzS+vj4eGl9165dF9zTxYyRHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4Dr7NM2ePbtjbdGiRaXrbtu2rep2LgpLly7ta/2xsbHuT8JnGNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAmus0/TqVOnOtb27t1buu61115bWh8aGiqtnzhxorTeZldccUXHWreprrt5/fXX+1o/m64ju+3nbB+zPTZp2WO2D9veW/yVT0AOoHHT2Y3/qaQ1Uyz/t4hYUfz9utq2AFSta9gj4jVJM3c/EoCk/k7QPWD7zWI3f16nJ9ketj1qe7SP9wLQp17DvkXSUkkrJB2R9ONOT4yIkYhYGREre3wvABXoKewRcTQizkbEOUk/kXRDtW0BqFpPYbe9cNLD70jiu4ZAy3W9zm57m6RbJM23fUjSjyTdYnuFpJB0QNL36muxHc6cOdOx9t5775Wue9ddd5XWX3755dL65s2bS+t1Wr58eWn9yiuvLK2XzcEeEb209Jlz5871tX42XcMeERumWPxsDb0AqBEflwWSIOxAEoQdSIKwA0kQdiAJ93v544LezB7cmw3QNddcU1p//PHHS+u33357ab3sZ6zrdvz48dJ6t38/ZdMu2+6pp/Pmzp1bWi+7XHoxi4gpNywjO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kwXX2FlixYkVp/aqrrhpMI1N4/vnn+1p/69atHWsbN27s67VnzeKX0KfCdXYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIILlS3QbcrnbvU2279/f22v3e1nrsfGmM5gMkZ2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiC6+yoVdlvw/f7u/FcR78wXUd224tt77L9ju23bf+gWD5k+xXb7xa38+pvF0CvprMb/6mkf4qIZZJulPR928skPSRpZ0RcLWln8RhAS3UNe0QciYg3ivunJI1LWiRpnaTzvzm0VdKdNfUIoAIXdMxue4mkb0r6vaQFEXGkKH0gaUGHdYYlDffRI4AKTPtsvO2vSXpB0g8j4k+TazHxq5VT/phkRIxExMqIWNlXpwD6Mq2w2/6KJoL+84jYXiw+anthUV8o6Vg9LQKownTOxlvSs5LGI2LzpNIOSZuK+5skvVR9e5jpIqK2P1yY6Ryz/42kv5f0lu29xbKHJT0l6Ze275f0vqTv1tIhgEp0DXtEvC6p06cfvlVtOwDqwsdlgSQIO5AEYQeSIOxAEoQdSIKvuKJWl156ac/rnjlzpsJOwMgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwnR21uvfeezvWTp48WbruE088UXE3uTGyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASXGdHrXbv3t2xtnnz5o41Sdq1a1fV7aTGyA4kQdiBJAg7kARhB5Ig7EAShB1IgrADSbjbPNe2F0v6maQFkkLSSET8u+3HJP2jpA+Lpz4cEb/u8lpMqg3ULCKmnHV5OmFfKGlhRLxhe66kPZLu1MR87Kcj4l+n2wRhB+rXKezTmZ/9iKQjxf1TtsclLaq2PQB1u6BjdttLJH1T0u+LRQ/YftP2c7bndVhn2Pao7dH+WgXQj6678Z890f6apP+W9C8Rsd32AknHNXEc/4QmdvXv6/Ia7MYDNev5mF2SbH9F0q8k/SYivvTthWLE/1VELO/yOoQdqFmnsHfdjbdtSc9KGp8c9OLE3XnfkTTWb5MA6jOds/GrJP2PpLcknSsWPyxpg6QVmtiNPyDpe8XJvLLXYmQHatbXbnxVCDtQv5534wFcHAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJDHrK5uOS3p/0eH6xrI3a2ltb+5LorVdV9vZXnQoD/T77l97cHo2IlY01UKKtvbW1L4neejWo3tiNB5Ig7EASTYd9pOH3L9PW3tral0RvvRpIb40eswMYnKZHdgADQtiBJBoJu+01tv9ge5/th5rooRPbB2y/ZXtv0/PTFXPoHbM9NmnZkO1XbL9b3E45x15DvT1m+3Cx7fbaXttQb4tt77L9ju23bf+gWN7otivpayDbbeDH7LYvkfRHSd+WdEjSbkkbIuKdgTbSge0DklZGROMfwLD9t5JOS/rZ+am1bD8t6UREPFX8j3JeRPxzS3p7TBc4jXdNvXWaZvwf1OC2q3L68140MbLfIGlfROyPiD9L+oWkdQ300XoR8ZqkE19YvE7S1uL+Vk38Yxm4Dr21QkQciYg3ivunJJ2fZrzRbVfS10A0EfZFkg5OenxI7ZrvPST91vYe28NNNzOFBZOm2fpA0oImm5lC12m8B+kL04y3Ztv1Mv15vzhB92WrIuI6SX8n6fvF7morxcQxWJuunW6RtFQTcwAekfTjJpspphl/QdIPI+JPk2tNbrsp+hrIdmsi7IclLZ70+OvFslaIiMPF7TFJL2risKNNjp6fQbe4PdZwP5+JiKMRcTYizkn6iRrcdsU04y9I+nlEbC8WN77tpuprUNutibDvlnS17W/Y/qqk9ZJ2NNDHl9ieU5w4ke05klarfVNR75C0qbi/SdJLDfbyOW2ZxrvTNONqeNs1Pv15RAz8T9JaTZyRf0/SI0300KGvKyX9b/H3dtO9Sdqmid26TzRxbuN+SX8paaekdyX9TtJQi3r7D01M7f2mJoK1sKHeVmliF/1NSXuLv7VNb7uSvgay3fi4LJAEJ+iAJAg7kARhB5Ig7EAShB1IgrADSRB2IIn/BwSyThmzraIZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model prediction: 4\n"
     ]
    }
   ],
   "source": [
    "# Predict 5 images from validation set.\n",
    "n_images = 5\n",
    "test_images = x_test[:n_images]\n",
    "predictions = neural_net(test_images)\n",
    "\n",
    "# Display image and model prediction.\n",
    "for i in range(n_images):\n",
    "    plt.imshow(np.reshape(test_images[i], [28, 28]), cmap='gray')\n",
    "    plt.show()\n",
    "    print(\"Model prediction: %i\" % np.argmax(predictions.numpy()[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
